{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.1"},"colab":{"name":"2-advanced-rnn.ipynb","provenance":[]}},"cells":[{"cell_type":"code","metadata":{"id":"Goc_l4YWzoO8","executionInfo":{"status":"ok","timestamp":1604888491693,"user_tz":420,"elapsed":2507,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["import numpy as np\n","\n","from sklearn.metrics import accuracy_score\n","from tensorflow.keras.datasets import reuters\n","from tensorflow.keras.preprocessing.sequence import pad_sequences\n","from tensorflow.keras.utils import to_categorical"],"execution_count":1,"outputs":[]},{"cell_type":"code","metadata":{"id":"7jBzWh0-zoPB","executionInfo":{"status":"ok","timestamp":1604888491694,"user_tz":420,"elapsed":2504,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["# parameters for data load\n","num_words = 30000\n","maxlen = 50\n","test_split = 0.3"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"b7cNDtj_zoPD","executionInfo":{"status":"ok","timestamp":1604888491915,"user_tz":420,"elapsed":2723,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"7da4d546-a818-41b5-e0d2-ac9e8de0445f","colab":{"base_uri":"https://localhost:8080/"}},"source":["(X_train, y_train), (X_test, y_test) = reuters.load_data(num_words = num_words, maxlen = maxlen, test_split = test_split)"],"execution_count":3,"outputs":[{"output_type":"stream","text":["Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/reuters.npz\n","2113536/2110848 [==============================] - 0s 0us/step\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"s1gOYZlgzoPG","executionInfo":{"status":"ok","timestamp":1604888491917,"user_tz":420,"elapsed":2721,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["# pad the sequences with zeros \n","# padding parameter is set to 'post' => 0's are appended to end of sequences\n","X_train = pad_sequences(X_train, padding = 'post')\n","X_test = pad_sequences(X_test, padding = 'post')\n","\n","X_train = np.array(X_train).reshape((X_train.shape[0], X_train.shape[1], 1))\n","X_test = np.array(X_test).reshape((X_test.shape[0], X_test.shape[1], 1))\n","\n","y_data = np.concatenate((y_train, y_test))\n","y_data = to_categorical(y_data)\n","y_train = y_data[:1395]\n","y_test = y_data[1395:]"],"execution_count":4,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"z7St1mCrzoPK"},"source":["## 1. Deep RNN\n","- RNNs can be made deep, with multiple layers, like CNNs or MLPs\n","- Beware that RNNs take long to train compared to CNNs"]},{"cell_type":"code","metadata":{"id":"TlbsuKsyzoPK","executionInfo":{"status":"ok","timestamp":1604888492109,"user_tz":420,"elapsed":2911,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["from tensorflow.keras.models import Sequential\n","from tensorflow.keras.layers import Dense, LSTM, Activation\n","from tensorflow.keras import optimizers\n","from tensorflow.keras.wrappers.scikit_learn import KerasClassifier"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"gCk03TBdzoPN","executionInfo":{"status":"ok","timestamp":1604888492110,"user_tz":420,"elapsed":2910,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["def deep_lstm():\n","    model = Sequential()\n","    model.add(LSTM(20, input_shape = (49,1), return_sequences = True))\n","    model.add(LSTM(20, return_sequences = True))\n","    model.add(LSTM(20, return_sequences = True))\n","    model.add(LSTM(20, return_sequences = False))\n","    model.add(Dense(46))\n","    model.add(Activation('softmax'))\n","    \n","    adam = optimizers.Adam(lr = 0.001)\n","    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n","    \n","    return model"],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"Jqc_RK0OzoPP","executionInfo":{"status":"ok","timestamp":1604888830577,"user_tz":420,"elapsed":341375,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"f3d43dff-b3bf-4547-e366-49e1bf4c1693","colab":{"base_uri":"https://localhost:8080/"}},"source":["model = KerasClassifier(build_fn = deep_lstm, epochs = 200, batch_size = 50, verbose = 1)\n","model.fit(X_train, y_train)"],"execution_count":7,"outputs":[{"output_type":"stream","text":["Epoch 1/200\n","28/28 [==============================] - 2s 55ms/step - loss: 3.2571 - accuracy: 0.6803\n","Epoch 2/200\n","28/28 [==============================] - 2s 55ms/step - loss: 2.0078 - accuracy: 0.7147\n","Epoch 3/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.3712 - accuracy: 0.7147\n","Epoch 4/200\n","28/28 [==============================] - 2s 55ms/step - loss: 1.2368 - accuracy: 0.7147\n","Epoch 5/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1963 - accuracy: 0.7147\n","Epoch 6/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1817 - accuracy: 0.7147\n","Epoch 7/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1727 - accuracy: 0.7147\n","Epoch 8/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1684 - accuracy: 0.7147\n","Epoch 9/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1651 - accuracy: 0.7147\n","Epoch 10/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1636 - accuracy: 0.7147\n","Epoch 11/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1619 - accuracy: 0.7147\n","Epoch 12/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1609 - accuracy: 0.7147\n","Epoch 13/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1595 - accuracy: 0.7147\n","Epoch 14/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1577 - accuracy: 0.7147\n","Epoch 15/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1561 - accuracy: 0.7147\n","Epoch 16/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1507 - accuracy: 0.7147\n","Epoch 17/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1489 - accuracy: 0.7147\n","Epoch 18/200\n","28/28 [==============================] - 2s 56ms/step - loss: 1.1211 - accuracy: 0.7147\n","Epoch 19/200\n","28/28 [==============================] - 2s 55ms/step - loss: 1.0693 - accuracy: 0.7147\n","Epoch 20/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.9498 - accuracy: 0.7634\n","Epoch 21/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.9254 - accuracy: 0.7785\n","Epoch 22/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.9162 - accuracy: 0.7735\n","Epoch 23/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.9946 - accuracy: 0.7577\n","Epoch 24/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.8852 - accuracy: 0.7842\n","Epoch 25/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.8602 - accuracy: 0.7964\n","Epoch 26/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.8550 - accuracy: 0.7835\n","Epoch 27/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.8602 - accuracy: 0.7964\n","Epoch 28/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.8319 - accuracy: 0.7907\n","Epoch 29/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.8289 - accuracy: 0.7950\n","Epoch 30/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.8390 - accuracy: 0.7914\n","Epoch 31/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.8254 - accuracy: 0.7943\n","Epoch 32/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.8153 - accuracy: 0.7907\n","Epoch 33/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.8220 - accuracy: 0.7885\n","Epoch 34/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.8317 - accuracy: 0.7921\n","Epoch 35/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.8220 - accuracy: 0.7885\n","Epoch 36/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.8227 - accuracy: 0.7957\n","Epoch 37/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.8129 - accuracy: 0.7943\n","Epoch 38/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.8075 - accuracy: 0.7978\n","Epoch 39/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.8138 - accuracy: 0.7921\n","Epoch 40/200\n","28/28 [==============================] - 2s 55ms/step - loss: 0.8111 - accuracy: 0.7957\n","Epoch 41/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.8638 - accuracy: 0.7692\n","Epoch 42/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.8259 - accuracy: 0.7964\n","Epoch 43/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.8072 - accuracy: 0.7971\n","Epoch 44/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.8022 - accuracy: 0.7964\n","Epoch 45/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7999 - accuracy: 0.7964\n","Epoch 46/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.8004 - accuracy: 0.7971\n","Epoch 47/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7932 - accuracy: 0.8014\n","Epoch 48/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7923 - accuracy: 0.7993\n","Epoch 49/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7998 - accuracy: 0.7964\n","Epoch 50/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7952 - accuracy: 0.7993\n","Epoch 51/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7877 - accuracy: 0.8029\n","Epoch 52/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7824 - accuracy: 0.8065\n","Epoch 53/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7768 - accuracy: 0.8029\n","Epoch 54/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7835 - accuracy: 0.8022\n","Epoch 55/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7793 - accuracy: 0.8043\n","Epoch 56/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7724 - accuracy: 0.8086\n","Epoch 57/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7661 - accuracy: 0.8050\n","Epoch 58/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7661 - accuracy: 0.8057\n","Epoch 59/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7615 - accuracy: 0.8072\n","Epoch 60/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7621 - accuracy: 0.8014\n","Epoch 61/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7613 - accuracy: 0.8065\n","Epoch 62/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7590 - accuracy: 0.8043\n","Epoch 63/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7564 - accuracy: 0.8057\n","Epoch 64/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7695 - accuracy: 0.8036\n","Epoch 65/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7504 - accuracy: 0.8057\n","Epoch 66/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7512 - accuracy: 0.8086\n","Epoch 67/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7483 - accuracy: 0.8057\n","Epoch 68/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7474 - accuracy: 0.8100\n","Epoch 69/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7435 - accuracy: 0.8100\n","Epoch 70/200\n","28/28 [==============================] - 2s 59ms/step - loss: 0.7491 - accuracy: 0.8108\n","Epoch 71/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.7448 - accuracy: 0.8093\n","Epoch 72/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7348 - accuracy: 0.8136\n","Epoch 73/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7455 - accuracy: 0.8100\n","Epoch 74/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7719 - accuracy: 0.8072\n","Epoch 75/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7364 - accuracy: 0.8100\n","Epoch 76/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7257 - accuracy: 0.8151\n","Epoch 77/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7255 - accuracy: 0.8186\n","Epoch 78/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7182 - accuracy: 0.8244\n","Epoch 79/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7140 - accuracy: 0.8208\n","Epoch 80/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7176 - accuracy: 0.8201\n","Epoch 81/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7326 - accuracy: 0.8172\n","Epoch 82/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7138 - accuracy: 0.8251\n","Epoch 83/200\n","28/28 [==============================] - 2s 59ms/step - loss: 0.7071 - accuracy: 0.8222\n","Epoch 84/200\n","28/28 [==============================] - 2s 59ms/step - loss: 0.7096 - accuracy: 0.8194\n","Epoch 85/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7001 - accuracy: 0.8301\n","Epoch 86/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7179 - accuracy: 0.8194\n","Epoch 87/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7038 - accuracy: 0.8287\n","Epoch 88/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7043 - accuracy: 0.8258\n","Epoch 89/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6877 - accuracy: 0.8337\n","Epoch 90/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6961 - accuracy: 0.8323\n","Epoch 91/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.7075 - accuracy: 0.8244\n","Epoch 92/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7241 - accuracy: 0.8265\n","Epoch 93/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6909 - accuracy: 0.8265\n","Epoch 94/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6932 - accuracy: 0.8315\n","Epoch 95/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7711 - accuracy: 0.8007\n","Epoch 96/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7441 - accuracy: 0.8072\n","Epoch 97/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7361 - accuracy: 0.8158\n","Epoch 98/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7119 - accuracy: 0.8229\n","Epoch 99/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7028 - accuracy: 0.8215\n","Epoch 100/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7111 - accuracy: 0.8194\n","Epoch 101/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6988 - accuracy: 0.8222\n","Epoch 102/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.7105 - accuracy: 0.8215\n","Epoch 103/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6946 - accuracy: 0.8208\n","Epoch 104/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.7046 - accuracy: 0.8215\n","Epoch 105/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6892 - accuracy: 0.8287\n","Epoch 106/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6885 - accuracy: 0.8301\n","Epoch 107/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6903 - accuracy: 0.8251\n","Epoch 108/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6861 - accuracy: 0.8251\n","Epoch 109/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6867 - accuracy: 0.8272\n","Epoch 110/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6895 - accuracy: 0.8265\n","Epoch 111/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6666 - accuracy: 0.8287\n","Epoch 112/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6689 - accuracy: 0.8301\n","Epoch 113/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6723 - accuracy: 0.8323\n","Epoch 114/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6676 - accuracy: 0.8323\n","Epoch 115/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6585 - accuracy: 0.8315\n","Epoch 116/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6479 - accuracy: 0.8366\n","Epoch 117/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6715 - accuracy: 0.8366\n","Epoch 118/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6715 - accuracy: 0.8308\n","Epoch 119/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6571 - accuracy: 0.8323\n","Epoch 120/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6483 - accuracy: 0.8380\n","Epoch 121/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6539 - accuracy: 0.8380\n","Epoch 122/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6575 - accuracy: 0.8301\n","Epoch 123/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6423 - accuracy: 0.8315\n","Epoch 124/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6469 - accuracy: 0.8358\n","Epoch 125/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6435 - accuracy: 0.8323\n","Epoch 126/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6243 - accuracy: 0.8473\n","Epoch 127/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6153 - accuracy: 0.8502\n","Epoch 128/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6220 - accuracy: 0.8480\n","Epoch 129/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6111 - accuracy: 0.8516\n","Epoch 130/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6357 - accuracy: 0.8416\n","Epoch 131/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6439 - accuracy: 0.8409\n","Epoch 132/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6228 - accuracy: 0.8473\n","Epoch 133/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6020 - accuracy: 0.8566\n","Epoch 134/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6063 - accuracy: 0.8530\n","Epoch 135/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6153 - accuracy: 0.8452\n","Epoch 136/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6043 - accuracy: 0.8516\n","Epoch 137/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5925 - accuracy: 0.8530\n","Epoch 138/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5908 - accuracy: 0.8581\n","Epoch 139/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5939 - accuracy: 0.8530\n","Epoch 140/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5934 - accuracy: 0.8523\n","Epoch 141/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5939 - accuracy: 0.8473\n","Epoch 142/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6026 - accuracy: 0.8545\n","Epoch 143/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6138 - accuracy: 0.8459\n","Epoch 144/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6089 - accuracy: 0.8473\n","Epoch 145/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.6015 - accuracy: 0.8509\n","Epoch 146/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5711 - accuracy: 0.8581\n","Epoch 147/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5854 - accuracy: 0.8530\n","Epoch 148/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.5760 - accuracy: 0.8616\n","Epoch 149/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5778 - accuracy: 0.8602\n","Epoch 150/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.5600 - accuracy: 0.8624\n","Epoch 151/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5655 - accuracy: 0.8595\n","Epoch 152/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5762 - accuracy: 0.8573\n","Epoch 153/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.5681 - accuracy: 0.8609\n","Epoch 154/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5564 - accuracy: 0.8631\n","Epoch 155/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5556 - accuracy: 0.8609\n","Epoch 156/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5577 - accuracy: 0.8616\n","Epoch 157/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5491 - accuracy: 0.8631\n","Epoch 158/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5526 - accuracy: 0.8609\n","Epoch 159/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5323 - accuracy: 0.8681\n","Epoch 160/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5284 - accuracy: 0.8717\n","Epoch 161/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5266 - accuracy: 0.8710\n","Epoch 162/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5413 - accuracy: 0.8667\n","Epoch 163/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.5366 - accuracy: 0.8667\n","Epoch 164/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.5359 - accuracy: 0.8652\n","Epoch 165/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.5902 - accuracy: 0.8523\n","Epoch 166/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6145 - accuracy: 0.8509\n","Epoch 167/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5761 - accuracy: 0.8573\n","Epoch 168/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.6041 - accuracy: 0.8452\n","Epoch 169/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5570 - accuracy: 0.8652\n","Epoch 170/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.5341 - accuracy: 0.8667\n","Epoch 171/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.5310 - accuracy: 0.8631\n","Epoch 172/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5235 - accuracy: 0.8652\n","Epoch 173/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5063 - accuracy: 0.8710\n","Epoch 174/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5246 - accuracy: 0.8638\n","Epoch 175/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5310 - accuracy: 0.8638\n","Epoch 176/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.5148 - accuracy: 0.8667\n","Epoch 177/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5079 - accuracy: 0.8688\n","Epoch 178/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.5026 - accuracy: 0.8695\n","Epoch 179/200\n","28/28 [==============================] - 2s 59ms/step - loss: 0.4983 - accuracy: 0.8746\n","Epoch 180/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.4910 - accuracy: 0.8724\n","Epoch 181/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5015 - accuracy: 0.8753\n","Epoch 182/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.4953 - accuracy: 0.8724\n","Epoch 183/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.4814 - accuracy: 0.8767\n","Epoch 184/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.5020 - accuracy: 0.8738\n","Epoch 185/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.4833 - accuracy: 0.8796\n","Epoch 186/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.4982 - accuracy: 0.8681\n","Epoch 187/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.5600 - accuracy: 0.8559\n","Epoch 188/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5243 - accuracy: 0.8659\n","Epoch 189/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.4823 - accuracy: 0.8760\n","Epoch 190/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.4962 - accuracy: 0.8681\n","Epoch 191/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.4946 - accuracy: 0.8724\n","Epoch 192/200\n","28/28 [==============================] - 2s 58ms/step - loss: 0.4742 - accuracy: 0.8753\n","Epoch 193/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.4931 - accuracy: 0.8710\n","Epoch 194/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.5047 - accuracy: 0.8659\n","Epoch 195/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.4969 - accuracy: 0.8688\n","Epoch 196/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.4743 - accuracy: 0.8724\n","Epoch 197/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.4811 - accuracy: 0.8703\n","Epoch 198/200\n","28/28 [==============================] - 2s 57ms/step - loss: 0.4694 - accuracy: 0.8767\n","Epoch 199/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.4641 - accuracy: 0.8781\n","Epoch 200/200\n","28/28 [==============================] - 2s 56ms/step - loss: 0.4972 - accuracy: 0.8717\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["<tensorflow.python.keras.callbacks.History at 0x7fc8c198e6d8>"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"code","metadata":{"id":"VSva5zQbzoPS","executionInfo":{"status":"ok","timestamp":1604888831917,"user_tz":420,"elapsed":342709,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"eab84f8b-0d15-42f1-e17c-8662161f77a1","colab":{"base_uri":"https://localhost:8080/"}},"source":["y_pred = model.predict(X_test)\n","y_test_ = np.argmax(y_test, axis = 1)"],"execution_count":8,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/wrappers/scikit_learn.py:241: Sequential.predict_classes (from tensorflow.python.keras.engine.sequential) is deprecated and will be removed after 2021-01-01.\n","Instructions for updating:\n","Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n","12/12 [==============================] - 0s 12ms/step\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"IX-j2A-OzoPU","executionInfo":{"status":"ok","timestamp":1604888831918,"user_tz":420,"elapsed":342705,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"7b5ac926-4ff4-4ed2-9ab4-b226572de048","colab":{"base_uri":"https://localhost:8080/"}},"source":["print(accuracy_score(y_pred, y_test_))"],"execution_count":9,"outputs":[{"output_type":"stream","text":["0.8497495826377296\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"lpNOjkjAzoPW"},"source":["## 2. Bidirectional RNN\n","- Bidirectional RNNs consider not only one-way influence of sequence, but also the other way\n","- Actually, they can be thought as building two separate RNNs, and merging them\\\n","<br>\n","<img src=\"http://d3kbpzbmcynnmx.cloudfront.net/wp-content/uploads/2015/09/bidirectional-rnn.png\" style=\"width: 400px\"/>\n","</br>"]},{"cell_type":"code","metadata":{"id":"_J4BVxxTzoPX","executionInfo":{"status":"ok","timestamp":1604888831919,"user_tz":420,"elapsed":342704,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["from tensorflow.keras.layers import Bidirectional"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"-fEcbhkezoPa","executionInfo":{"status":"ok","timestamp":1604888831920,"user_tz":420,"elapsed":342703,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["def bidirectional_lstm():\n","    model = Sequential()\n","    model.add(Bidirectional(LSTM(20, return_sequences = False), input_shape = (49,1)))\n","    model.add(Dense(46))\n","    model.add(Activation('softmax'))\n","    \n","    adam = optimizers.Adam(lr = 0.001)\n","    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n","    \n","    return model"],"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"id":"UdOjohvWzoPd","executionInfo":{"status":"ok","timestamp":1604888944987,"user_tz":420,"elapsed":455768,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"c0c0f6a6-2c86-4427-f92e-a6daf135c8ec","colab":{"base_uri":"https://localhost:8080/"}},"source":["model = KerasClassifier(build_fn = bidirectional_lstm, epochs = 200, batch_size = 50, verbose = 1)\n","model.fit(X_train, y_train)"],"execution_count":12,"outputs":[{"output_type":"stream","text":["Epoch 1/200\n","28/28 [==============================] - 1s 19ms/step - loss: 3.5629 - accuracy: 0.2330\n","Epoch 2/200\n","28/28 [==============================] - 1s 19ms/step - loss: 2.4295 - accuracy: 0.7039\n","Epoch 3/200\n","28/28 [==============================] - 1s 18ms/step - loss: 1.3877 - accuracy: 0.7147\n","Epoch 4/200\n","28/28 [==============================] - 1s 19ms/step - loss: 1.1779 - accuracy: 0.7147\n","Epoch 5/200\n","28/28 [==============================] - 1s 20ms/step - loss: 1.1300 - accuracy: 0.7147\n","Epoch 6/200\n","28/28 [==============================] - 1s 20ms/step - loss: 1.0874 - accuracy: 0.7147\n","Epoch 7/200\n","28/28 [==============================] - 1s 20ms/step - loss: 1.0424 - accuracy: 0.7147\n","Epoch 8/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.9979 - accuracy: 0.7161\n","Epoch 9/200\n","28/28 [==============================] - 0s 18ms/step - loss: 0.9545 - accuracy: 0.7226\n","Epoch 10/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.9078 - accuracy: 0.7548\n","Epoch 11/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.8690 - accuracy: 0.7842\n","Epoch 12/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.8465 - accuracy: 0.8007\n","Epoch 13/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.8270 - accuracy: 0.8043\n","Epoch 14/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.7939 - accuracy: 0.8172\n","Epoch 15/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.7826 - accuracy: 0.8172\n","Epoch 16/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7868 - accuracy: 0.8237\n","Epoch 17/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.7662 - accuracy: 0.8136\n","Epoch 18/200\n","28/28 [==============================] - 0s 18ms/step - loss: 0.7593 - accuracy: 0.8194\n","Epoch 19/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.7503 - accuracy: 0.8186\n","Epoch 20/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7515 - accuracy: 0.8186\n","Epoch 21/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7463 - accuracy: 0.8158\n","Epoch 22/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7387 - accuracy: 0.8229\n","Epoch 23/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.7327 - accuracy: 0.8244\n","Epoch 24/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.7353 - accuracy: 0.8251\n","Epoch 25/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7292 - accuracy: 0.8258\n","Epoch 26/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.7272 - accuracy: 0.8280\n","Epoch 27/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7265 - accuracy: 0.8308\n","Epoch 28/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7183 - accuracy: 0.8294\n","Epoch 29/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7147 - accuracy: 0.8280\n","Epoch 30/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.7153 - accuracy: 0.8287\n","Epoch 31/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7120 - accuracy: 0.8330\n","Epoch 32/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.7059 - accuracy: 0.8323\n","Epoch 33/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.7002 - accuracy: 0.8358\n","Epoch 34/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6979 - accuracy: 0.8351\n","Epoch 35/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.7009 - accuracy: 0.8358\n","Epoch 36/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6914 - accuracy: 0.8373\n","Epoch 37/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6925 - accuracy: 0.8358\n","Epoch 38/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6900 - accuracy: 0.8401\n","Epoch 39/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6847 - accuracy: 0.8373\n","Epoch 40/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6809 - accuracy: 0.8387\n","Epoch 41/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6798 - accuracy: 0.8416\n","Epoch 42/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6749 - accuracy: 0.8401\n","Epoch 43/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6727 - accuracy: 0.8430\n","Epoch 44/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6749 - accuracy: 0.8444\n","Epoch 45/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6688 - accuracy: 0.8444\n","Epoch 46/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6685 - accuracy: 0.8423\n","Epoch 47/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6713 - accuracy: 0.8401\n","Epoch 48/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6626 - accuracy: 0.8430\n","Epoch 49/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6616 - accuracy: 0.8430\n","Epoch 50/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6613 - accuracy: 0.8423\n","Epoch 51/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6551 - accuracy: 0.8459\n","Epoch 52/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6558 - accuracy: 0.8430\n","Epoch 53/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6534 - accuracy: 0.8430\n","Epoch 54/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6493 - accuracy: 0.8480\n","Epoch 55/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6461 - accuracy: 0.8430\n","Epoch 56/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6528 - accuracy: 0.8473\n","Epoch 57/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6467 - accuracy: 0.8495\n","Epoch 58/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6408 - accuracy: 0.8487\n","Epoch 59/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6415 - accuracy: 0.8495\n","Epoch 60/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6402 - accuracy: 0.8459\n","Epoch 61/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6452 - accuracy: 0.8495\n","Epoch 62/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6531 - accuracy: 0.8423\n","Epoch 63/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6366 - accuracy: 0.8487\n","Epoch 64/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6290 - accuracy: 0.8466\n","Epoch 65/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6385 - accuracy: 0.8444\n","Epoch 66/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6311 - accuracy: 0.8473\n","Epoch 67/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6250 - accuracy: 0.8495\n","Epoch 68/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6230 - accuracy: 0.8495\n","Epoch 69/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6244 - accuracy: 0.8466\n","Epoch 70/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6213 - accuracy: 0.8452\n","Epoch 71/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6195 - accuracy: 0.8495\n","Epoch 72/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6140 - accuracy: 0.8495\n","Epoch 73/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6196 - accuracy: 0.8523\n","Epoch 74/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6133 - accuracy: 0.8495\n","Epoch 75/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6196 - accuracy: 0.8495\n","Epoch 76/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6080 - accuracy: 0.8495\n","Epoch 77/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6659 - accuracy: 0.8495\n","Epoch 78/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.6372 - accuracy: 0.8452\n","Epoch 79/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6117 - accuracy: 0.8473\n","Epoch 80/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6262 - accuracy: 0.8473\n","Epoch 81/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6083 - accuracy: 0.8495\n","Epoch 82/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6018 - accuracy: 0.8452\n","Epoch 83/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6017 - accuracy: 0.8487\n","Epoch 84/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5996 - accuracy: 0.8502\n","Epoch 85/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5965 - accuracy: 0.8495\n","Epoch 86/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5942 - accuracy: 0.8509\n","Epoch 87/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5936 - accuracy: 0.8516\n","Epoch 88/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5888 - accuracy: 0.8538\n","Epoch 89/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5888 - accuracy: 0.8487\n","Epoch 90/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.6033 - accuracy: 0.8480\n","Epoch 91/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.6041 - accuracy: 0.8509\n","Epoch 92/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5911 - accuracy: 0.8552\n","Epoch 93/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5912 - accuracy: 0.8495\n","Epoch 94/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5844 - accuracy: 0.8509\n","Epoch 95/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5861 - accuracy: 0.8523\n","Epoch 96/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5828 - accuracy: 0.8495\n","Epoch 97/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5843 - accuracy: 0.8516\n","Epoch 98/200\n","28/28 [==============================] - 0s 18ms/step - loss: 0.5762 - accuracy: 0.8516\n","Epoch 99/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5747 - accuracy: 0.8573\n","Epoch 100/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5727 - accuracy: 0.8538\n","Epoch 101/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5708 - accuracy: 0.8552\n","Epoch 102/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5664 - accuracy: 0.8566\n","Epoch 103/200\n","28/28 [==============================] - 0s 18ms/step - loss: 0.5648 - accuracy: 0.8552\n","Epoch 104/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5662 - accuracy: 0.8552\n","Epoch 105/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5694 - accuracy: 0.8530\n","Epoch 106/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5620 - accuracy: 0.8559\n","Epoch 107/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5579 - accuracy: 0.8559\n","Epoch 108/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5619 - accuracy: 0.8573\n","Epoch 109/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5574 - accuracy: 0.8559\n","Epoch 110/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5565 - accuracy: 0.8602\n","Epoch 111/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5630 - accuracy: 0.8573\n","Epoch 112/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5556 - accuracy: 0.8545\n","Epoch 113/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5544 - accuracy: 0.8552\n","Epoch 114/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5518 - accuracy: 0.8566\n","Epoch 115/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5529 - accuracy: 0.8595\n","Epoch 116/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5465 - accuracy: 0.8581\n","Epoch 117/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5527 - accuracy: 0.8616\n","Epoch 118/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5522 - accuracy: 0.8523\n","Epoch 119/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5481 - accuracy: 0.8609\n","Epoch 120/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5756 - accuracy: 0.8559\n","Epoch 121/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5430 - accuracy: 0.8616\n","Epoch 122/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5407 - accuracy: 0.8588\n","Epoch 123/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5373 - accuracy: 0.8624\n","Epoch 124/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5324 - accuracy: 0.8616\n","Epoch 125/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5322 - accuracy: 0.8609\n","Epoch 126/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.5298 - accuracy: 0.8659\n","Epoch 127/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5368 - accuracy: 0.8609\n","Epoch 128/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.5358 - accuracy: 0.8638\n","Epoch 129/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5267 - accuracy: 0.8624\n","Epoch 130/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5276 - accuracy: 0.8638\n","Epoch 131/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5272 - accuracy: 0.8681\n","Epoch 132/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5328 - accuracy: 0.8652\n","Epoch 133/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5239 - accuracy: 0.8624\n","Epoch 134/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5193 - accuracy: 0.8674\n","Epoch 135/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.5169 - accuracy: 0.8674\n","Epoch 136/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5202 - accuracy: 0.8645\n","Epoch 137/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5183 - accuracy: 0.8624\n","Epoch 138/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5136 - accuracy: 0.8645\n","Epoch 139/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5196 - accuracy: 0.8681\n","Epoch 140/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5126 - accuracy: 0.8674\n","Epoch 141/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5169 - accuracy: 0.8659\n","Epoch 142/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5134 - accuracy: 0.8681\n","Epoch 143/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5097 - accuracy: 0.8717\n","Epoch 144/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5189 - accuracy: 0.8638\n","Epoch 145/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5088 - accuracy: 0.8681\n","Epoch 146/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5107 - accuracy: 0.8695\n","Epoch 147/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5028 - accuracy: 0.8703\n","Epoch 148/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5038 - accuracy: 0.8688\n","Epoch 149/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5015 - accuracy: 0.8667\n","Epoch 150/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4980 - accuracy: 0.8717\n","Epoch 151/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5192 - accuracy: 0.8659\n","Epoch 152/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5207 - accuracy: 0.8674\n","Epoch 153/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5167 - accuracy: 0.8645\n","Epoch 154/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5075 - accuracy: 0.8674\n","Epoch 155/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5098 - accuracy: 0.8659\n","Epoch 156/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.5030 - accuracy: 0.8652\n","Epoch 157/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5065 - accuracy: 0.8681\n","Epoch 158/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5014 - accuracy: 0.8703\n","Epoch 159/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5052 - accuracy: 0.8710\n","Epoch 160/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.5008 - accuracy: 0.8724\n","Epoch 161/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.5234 - accuracy: 0.8674\n","Epoch 162/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.4956 - accuracy: 0.8717\n","Epoch 163/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.4943 - accuracy: 0.8695\n","Epoch 164/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4971 - accuracy: 0.8695\n","Epoch 165/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4935 - accuracy: 0.8667\n","Epoch 166/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.4877 - accuracy: 0.8738\n","Epoch 167/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4868 - accuracy: 0.8731\n","Epoch 168/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.4883 - accuracy: 0.8717\n","Epoch 169/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4841 - accuracy: 0.8717\n","Epoch 170/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4835 - accuracy: 0.8724\n","Epoch 171/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4872 - accuracy: 0.8724\n","Epoch 172/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4825 - accuracy: 0.8710\n","Epoch 173/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4759 - accuracy: 0.8717\n","Epoch 174/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4762 - accuracy: 0.8738\n","Epoch 175/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4737 - accuracy: 0.8724\n","Epoch 176/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4738 - accuracy: 0.8781\n","Epoch 177/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.4722 - accuracy: 0.8695\n","Epoch 178/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.4763 - accuracy: 0.8760\n","Epoch 179/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4765 - accuracy: 0.8753\n","Epoch 180/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4802 - accuracy: 0.8746\n","Epoch 181/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4669 - accuracy: 0.8731\n","Epoch 182/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.4724 - accuracy: 0.8753\n","Epoch 183/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4647 - accuracy: 0.8724\n","Epoch 184/200\n","28/28 [==============================] - 1s 18ms/step - loss: 0.4633 - accuracy: 0.8724\n","Epoch 185/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4684 - accuracy: 0.8753\n","Epoch 186/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4636 - accuracy: 0.8746\n","Epoch 187/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4597 - accuracy: 0.8774\n","Epoch 188/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4807 - accuracy: 0.8774\n","Epoch 189/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4663 - accuracy: 0.8760\n","Epoch 190/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4647 - accuracy: 0.8767\n","Epoch 191/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4705 - accuracy: 0.8731\n","Epoch 192/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4669 - accuracy: 0.8724\n","Epoch 193/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4581 - accuracy: 0.8789\n","Epoch 194/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4513 - accuracy: 0.8789\n","Epoch 195/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4520 - accuracy: 0.8767\n","Epoch 196/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.4520 - accuracy: 0.8789\n","Epoch 197/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4469 - accuracy: 0.8781\n","Epoch 198/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.4525 - accuracy: 0.8832\n","Epoch 199/200\n","28/28 [==============================] - 1s 20ms/step - loss: 0.4486 - accuracy: 0.8817\n","Epoch 200/200\n","28/28 [==============================] - 1s 19ms/step - loss: 0.4524 - accuracy: 0.8746\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["<tensorflow.python.keras.callbacks.History at 0x7fc8c1294a90>"]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"qCYmBlZYzoPg","executionInfo":{"status":"ok","timestamp":1604888945675,"user_tz":420,"elapsed":456451,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"fb7f0f2a-da45-4fb5-8ba9-d53911bfc53a","colab":{"base_uri":"https://localhost:8080/"}},"source":["y_pred = model.predict(X_test)\n","y_test_ = np.argmax(y_test, axis = 1)"],"execution_count":13,"outputs":[{"output_type":"stream","text":["12/12 [==============================] - 0s 5ms/step\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"CdzkK9QqzoPi","executionInfo":{"status":"ok","timestamp":1604888945676,"user_tz":420,"elapsed":456447,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"8977d278-0eed-4461-eeed-918c026a67c9","colab":{"base_uri":"https://localhost:8080/"}},"source":["print(accuracy_score(y_pred, y_test_))"],"execution_count":14,"outputs":[{"output_type":"stream","text":["0.8430717863105175\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"ZwF6CRwyzoPl"},"source":["## 3. Deep Bidirectional RNN\n","- Bidirectional RNNs can be stacked\n","\n","<img src=\"http://www.wildml.com/wp-content/uploads/2015/09/Screen-Shot-2015-09-16-at-2.21.51-PM-272x300.png\" style=\"width: 300px\"/>"]},{"cell_type":"code","metadata":{"id":"C7S9jHN6zoPl","executionInfo":{"status":"ok","timestamp":1604888945676,"user_tz":420,"elapsed":456445,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["def deep_bidirectional_lstm():\n","    model = Sequential()\n","    model.add(Bidirectional(LSTM(10, return_sequences = True), input_shape = (49,1)))\n","    model.add(Bidirectional(LSTM(10, return_sequences = True)))\n","    model.add(Bidirectional(LSTM(10, return_sequences = True)))\n","    model.add(Bidirectional(LSTM(10, return_sequences = False)))\n","    model.add(Dense(46))\n","    model.add(Activation('softmax'))\n","    \n","    adam = optimizers.Adam(lr = 0.001)\n","    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n","    \n","    return model"],"execution_count":15,"outputs":[]},{"cell_type":"code","metadata":{"id":"joNfQvhezoPn","executionInfo":{"status":"ok","timestamp":1604889378739,"user_tz":420,"elapsed":889506,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"f751bfcc-3733-4ffd-81be-02c8b0503fc7","colab":{"base_uri":"https://localhost:8080/"}},"source":["model = KerasClassifier(build_fn = deep_bidirectional_lstm, epochs = 200, batch_size = 50, verbose = 1)\n","model.fit(X_train, y_train)"],"execution_count":16,"outputs":[{"output_type":"stream","text":["Epoch 1/200\n","28/28 [==============================] - 2s 72ms/step - loss: 3.5133 - accuracy: 0.2330\n","Epoch 2/200\n","28/28 [==============================] - 2s 72ms/step - loss: 2.4088 - accuracy: 0.7147\n","Epoch 3/200\n","28/28 [==============================] - 2s 71ms/step - loss: 1.5577 - accuracy: 0.7147\n","Epoch 4/200\n","28/28 [==============================] - 2s 71ms/step - loss: 1.2687 - accuracy: 0.7147\n","Epoch 5/200\n","28/28 [==============================] - 2s 74ms/step - loss: 1.2092 - accuracy: 0.7147\n","Epoch 6/200\n","28/28 [==============================] - 2s 73ms/step - loss: 1.1894 - accuracy: 0.7147\n","Epoch 7/200\n","28/28 [==============================] - 2s 72ms/step - loss: 1.1794 - accuracy: 0.7147\n","Epoch 8/200\n","28/28 [==============================] - 2s 73ms/step - loss: 1.1736 - accuracy: 0.7147\n","Epoch 9/200\n","28/28 [==============================] - 2s 72ms/step - loss: 1.1697 - accuracy: 0.7147\n","Epoch 10/200\n","28/28 [==============================] - 2s 73ms/step - loss: 1.1661 - accuracy: 0.7147\n","Epoch 11/200\n","28/28 [==============================] - 2s 72ms/step - loss: 1.1635 - accuracy: 0.7147\n","Epoch 12/200\n","28/28 [==============================] - 2s 72ms/step - loss: 1.1609 - accuracy: 0.7147\n","Epoch 13/200\n","28/28 [==============================] - 2s 73ms/step - loss: 1.1560 - accuracy: 0.7147\n","Epoch 14/200\n","28/28 [==============================] - 2s 72ms/step - loss: 1.0773 - accuracy: 0.7147\n","Epoch 15/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.9836 - accuracy: 0.7147\n","Epoch 16/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.9418 - accuracy: 0.7140\n","Epoch 17/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.8977 - accuracy: 0.7778\n","Epoch 18/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.8787 - accuracy: 0.7943\n","Epoch 19/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.8653 - accuracy: 0.7935\n","Epoch 20/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.8539 - accuracy: 0.7943\n","Epoch 21/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.8418 - accuracy: 0.7943\n","Epoch 22/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.8332 - accuracy: 0.7971\n","Epoch 23/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.8271 - accuracy: 0.7907\n","Epoch 24/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.8176 - accuracy: 0.8007\n","Epoch 25/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.8086 - accuracy: 0.8014\n","Epoch 26/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.8028 - accuracy: 0.8007\n","Epoch 27/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.8004 - accuracy: 0.7993\n","Epoch 28/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.7932 - accuracy: 0.8029\n","Epoch 29/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7894 - accuracy: 0.8050\n","Epoch 30/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7913 - accuracy: 0.8050\n","Epoch 31/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.7848 - accuracy: 0.8079\n","Epoch 32/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.7792 - accuracy: 0.8057\n","Epoch 33/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.7764 - accuracy: 0.8100\n","Epoch 34/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.7656 - accuracy: 0.8129\n","Epoch 35/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7578 - accuracy: 0.8079\n","Epoch 36/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7648 - accuracy: 0.8108\n","Epoch 37/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.7540 - accuracy: 0.8079\n","Epoch 38/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7611 - accuracy: 0.8108\n","Epoch 39/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.7560 - accuracy: 0.8007\n","Epoch 40/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7528 - accuracy: 0.8079\n","Epoch 41/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.7451 - accuracy: 0.8115\n","Epoch 42/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7374 - accuracy: 0.8115\n","Epoch 43/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7349 - accuracy: 0.8179\n","Epoch 44/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7354 - accuracy: 0.8172\n","Epoch 45/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.7253 - accuracy: 0.8151\n","Epoch 46/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7142 - accuracy: 0.8215\n","Epoch 47/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.7200 - accuracy: 0.8158\n","Epoch 48/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.7167 - accuracy: 0.8229\n","Epoch 49/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.7147 - accuracy: 0.8237\n","Epoch 50/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.7087 - accuracy: 0.8229\n","Epoch 51/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.7134 - accuracy: 0.8194\n","Epoch 52/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.7105 - accuracy: 0.8158\n","Epoch 53/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6957 - accuracy: 0.8272\n","Epoch 54/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.7123 - accuracy: 0.8186\n","Epoch 55/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6950 - accuracy: 0.8179\n","Epoch 56/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.6921 - accuracy: 0.8201\n","Epoch 57/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6893 - accuracy: 0.8244\n","Epoch 58/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6852 - accuracy: 0.8258\n","Epoch 59/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6760 - accuracy: 0.8287\n","Epoch 60/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6835 - accuracy: 0.8251\n","Epoch 61/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6747 - accuracy: 0.8251\n","Epoch 62/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6887 - accuracy: 0.8201\n","Epoch 63/200\n","28/28 [==============================] - 2s 74ms/step - loss: 0.6982 - accuracy: 0.8165\n","Epoch 64/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6759 - accuracy: 0.8215\n","Epoch 65/200\n","28/28 [==============================] - 2s 74ms/step - loss: 0.6791 - accuracy: 0.8186\n","Epoch 66/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.6900 - accuracy: 0.8237\n","Epoch 67/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.6595 - accuracy: 0.8301\n","Epoch 68/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.6590 - accuracy: 0.8301\n","Epoch 69/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6572 - accuracy: 0.8251\n","Epoch 70/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6466 - accuracy: 0.8301\n","Epoch 71/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6389 - accuracy: 0.8351\n","Epoch 72/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6444 - accuracy: 0.8280\n","Epoch 73/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6393 - accuracy: 0.8330\n","Epoch 74/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6428 - accuracy: 0.8323\n","Epoch 75/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6344 - accuracy: 0.8344\n","Epoch 76/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6301 - accuracy: 0.8366\n","Epoch 77/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.6845 - accuracy: 0.8315\n","Epoch 78/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6369 - accuracy: 0.8366\n","Epoch 79/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.6153 - accuracy: 0.8573\n","Epoch 80/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6271 - accuracy: 0.8473\n","Epoch 81/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6139 - accuracy: 0.8573\n","Epoch 82/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6045 - accuracy: 0.8588\n","Epoch 83/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6437 - accuracy: 0.8444\n","Epoch 84/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6195 - accuracy: 0.8495\n","Epoch 85/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6358 - accuracy: 0.8487\n","Epoch 86/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6098 - accuracy: 0.8588\n","Epoch 87/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6083 - accuracy: 0.8609\n","Epoch 88/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5986 - accuracy: 0.8573\n","Epoch 89/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5898 - accuracy: 0.8609\n","Epoch 90/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6008 - accuracy: 0.8602\n","Epoch 91/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5977 - accuracy: 0.8530\n","Epoch 92/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5906 - accuracy: 0.8581\n","Epoch 93/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6219 - accuracy: 0.8509\n","Epoch 94/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5876 - accuracy: 0.8638\n","Epoch 95/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5967 - accuracy: 0.8545\n","Epoch 96/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5729 - accuracy: 0.8645\n","Epoch 97/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5812 - accuracy: 0.8609\n","Epoch 98/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6048 - accuracy: 0.8595\n","Epoch 99/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5758 - accuracy: 0.8681\n","Epoch 100/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6563 - accuracy: 0.8280\n","Epoch 101/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.6386 - accuracy: 0.8466\n","Epoch 102/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5979 - accuracy: 0.8595\n","Epoch 103/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5787 - accuracy: 0.8667\n","Epoch 104/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5733 - accuracy: 0.8602\n","Epoch 105/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5644 - accuracy: 0.8681\n","Epoch 106/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5717 - accuracy: 0.8609\n","Epoch 107/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5707 - accuracy: 0.8652\n","Epoch 108/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.6042 - accuracy: 0.8473\n","Epoch 109/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5665 - accuracy: 0.8710\n","Epoch 110/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5601 - accuracy: 0.8688\n","Epoch 111/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5473 - accuracy: 0.8681\n","Epoch 112/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5410 - accuracy: 0.8746\n","Epoch 113/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5443 - accuracy: 0.8710\n","Epoch 114/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.5334 - accuracy: 0.8753\n","Epoch 115/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5407 - accuracy: 0.8695\n","Epoch 116/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5284 - accuracy: 0.8746\n","Epoch 117/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5318 - accuracy: 0.8767\n","Epoch 118/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5289 - accuracy: 0.8731\n","Epoch 119/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5303 - accuracy: 0.8746\n","Epoch 120/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5780 - accuracy: 0.8602\n","Epoch 121/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.6850 - accuracy: 0.8351\n","Epoch 122/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5603 - accuracy: 0.8717\n","Epoch 123/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5365 - accuracy: 0.8710\n","Epoch 124/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5582 - accuracy: 0.8681\n","Epoch 125/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5471 - accuracy: 0.8703\n","Epoch 126/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5326 - accuracy: 0.8710\n","Epoch 127/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5303 - accuracy: 0.8781\n","Epoch 128/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5416 - accuracy: 0.8695\n","Epoch 129/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5508 - accuracy: 0.8588\n","Epoch 130/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5710 - accuracy: 0.8595\n","Epoch 131/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5244 - accuracy: 0.8717\n","Epoch 132/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5284 - accuracy: 0.8746\n","Epoch 133/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5182 - accuracy: 0.8760\n","Epoch 134/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5147 - accuracy: 0.8796\n","Epoch 135/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5040 - accuracy: 0.8832\n","Epoch 136/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5060 - accuracy: 0.8803\n","Epoch 137/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5090 - accuracy: 0.8824\n","Epoch 138/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5035 - accuracy: 0.8824\n","Epoch 139/200\n","28/28 [==============================] - 2s 74ms/step - loss: 0.5108 - accuracy: 0.8753\n","Epoch 140/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4998 - accuracy: 0.8817\n","Epoch 141/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4954 - accuracy: 0.8796\n","Epoch 142/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5253 - accuracy: 0.8710\n","Epoch 143/200\n","28/28 [==============================] - 2s 74ms/step - loss: 0.5114 - accuracy: 0.8753\n","Epoch 144/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4904 - accuracy: 0.8839\n","Epoch 145/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5036 - accuracy: 0.8810\n","Epoch 146/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4858 - accuracy: 0.8860\n","Epoch 147/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4826 - accuracy: 0.8810\n","Epoch 148/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4864 - accuracy: 0.8753\n","Epoch 149/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.5271 - accuracy: 0.8674\n","Epoch 150/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4850 - accuracy: 0.8860\n","Epoch 151/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4713 - accuracy: 0.8867\n","Epoch 152/200\n","28/28 [==============================] - 2s 74ms/step - loss: 0.4682 - accuracy: 0.8875\n","Epoch 153/200\n","28/28 [==============================] - 2s 71ms/step - loss: 0.4768 - accuracy: 0.8853\n","Epoch 154/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4851 - accuracy: 0.8817\n","Epoch 155/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4982 - accuracy: 0.8781\n","Epoch 156/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.5015 - accuracy: 0.8774\n","Epoch 157/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4802 - accuracy: 0.8832\n","Epoch 158/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4767 - accuracy: 0.8781\n","Epoch 159/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4750 - accuracy: 0.8846\n","Epoch 160/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4818 - accuracy: 0.8846\n","Epoch 161/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4758 - accuracy: 0.8839\n","Epoch 162/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4676 - accuracy: 0.8860\n","Epoch 163/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4766 - accuracy: 0.8796\n","Epoch 164/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4659 - accuracy: 0.8896\n","Epoch 165/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4768 - accuracy: 0.8853\n","Epoch 166/200\n","28/28 [==============================] - 2s 74ms/step - loss: 0.4671 - accuracy: 0.8867\n","Epoch 167/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4475 - accuracy: 0.8925\n","Epoch 168/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4467 - accuracy: 0.8961\n","Epoch 169/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4820 - accuracy: 0.8860\n","Epoch 170/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4570 - accuracy: 0.8875\n","Epoch 171/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4395 - accuracy: 0.8946\n","Epoch 172/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4453 - accuracy: 0.8932\n","Epoch 173/200\n","28/28 [==============================] - 2s 74ms/step - loss: 0.4568 - accuracy: 0.8889\n","Epoch 174/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4381 - accuracy: 0.8953\n","Epoch 175/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4698 - accuracy: 0.8853\n","Epoch 176/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4438 - accuracy: 0.8953\n","Epoch 177/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4419 - accuracy: 0.8925\n","Epoch 178/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4792 - accuracy: 0.8853\n","Epoch 179/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4409 - accuracy: 0.8961\n","Epoch 180/200\n","28/28 [==============================] - 2s 74ms/step - loss: 0.4312 - accuracy: 0.8918\n","Epoch 181/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4277 - accuracy: 0.8989\n","Epoch 182/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4518 - accuracy: 0.8918\n","Epoch 183/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4368 - accuracy: 0.8932\n","Epoch 184/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4207 - accuracy: 0.8968\n","Epoch 185/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4497 - accuracy: 0.8875\n","Epoch 186/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4348 - accuracy: 0.8975\n","Epoch 187/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4240 - accuracy: 0.8982\n","Epoch 188/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4326 - accuracy: 0.8953\n","Epoch 189/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4376 - accuracy: 0.8939\n","Epoch 190/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4438 - accuracy: 0.8875\n","Epoch 191/200\n","28/28 [==============================] - 2s 74ms/step - loss: 0.4180 - accuracy: 0.8968\n","Epoch 192/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4135 - accuracy: 0.8989\n","Epoch 193/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4109 - accuracy: 0.9004\n","Epoch 194/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4135 - accuracy: 0.8996\n","Epoch 195/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4408 - accuracy: 0.8910\n","Epoch 196/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.4267 - accuracy: 0.8867\n","Epoch 197/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4029 - accuracy: 0.8968\n","Epoch 198/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.4021 - accuracy: 0.9011\n","Epoch 199/200\n","28/28 [==============================] - 2s 73ms/step - loss: 0.3923 - accuracy: 0.9032\n","Epoch 200/200\n","28/28 [==============================] - 2s 72ms/step - loss: 0.3998 - accuracy: 0.9018\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["<tensorflow.python.keras.callbacks.History at 0x7fc8b63cb8d0>"]},"metadata":{"tags":[]},"execution_count":16}]},{"cell_type":"code","metadata":{"id":"P9am0tiXzoPr","executionInfo":{"status":"ok","timestamp":1604889381187,"user_tz":420,"elapsed":891949,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"4daecb03-3da8-4600-e84b-46a249d96b8e","colab":{"base_uri":"https://localhost:8080/"}},"source":["y_pred = model.predict(X_test)\n","y_test_ = np.argmax(y_test, axis = 1)"],"execution_count":17,"outputs":[{"output_type":"stream","text":["12/12 [==============================] - 0s 14ms/step\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"x77CTIj2zoPt","executionInfo":{"status":"ok","timestamp":1604889381188,"user_tz":420,"elapsed":891945,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"8e10d6b2-fe19-4ce1-d84b-edb9c677580a","colab":{"base_uri":"https://localhost:8080/"}},"source":["print(accuracy_score(y_pred, y_test_))"],"execution_count":18,"outputs":[{"output_type":"stream","text":["0.8447412353923205\n"],"name":"stdout"}]}]}